#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>

#include <darknet/activation_layer.h>
#include <darknet/activations.h>

#include <darknet/avgpool_layer.h>
#include <darknet/batchnorm_layer.h>
#include <darknet/blas.h>
#include <darknet/connected_layer.h>
#include <darknet/convolutional_layer.h>
#include <darknet/cost_layer.h>
#include <darknet/crnn_layer.h>
#include <darknet/crop_layer.h>
#include <darknet/detection_layer.h>
#include <darknet/dropout_layer.h>
#include <darknet/gru_layer.h>
#include <darknet/list.h>
#include <darknet/local_layer.h>
#include <darknet/lstm_layer.h>
#include <darknet/conv_lstm_layer.h>
#include <darknet/maxpool_layer.h>
#include <darknet/normalization_layer.h>
#include <darknet/option_list.h>
#include <darknet/parser.h>
#include <darknet/region_layer.h>
#include <darknet/reorg_layer.h>
#include <darknet/reorg_old_layer.h>
#include <darknet/rnn_layer.h>
#include <darknet/route_layer.h>
#include <darknet/shortcut_layer.h>
#include <darknet/scale_channels_layer.h>
#include <darknet/softmax_layer.h>
#include <darknet/utils.h>
#include <darknet/upsample_layer.h>
#include <darknet/version.h>
#include <darknet/yolo_layer.h>

typedef struct
{
    char *type;
    list *options;
} section;

list *read_cfg(char *filename);

LAYER_TYPE string_to_layer_type(char *type)
{

    if (strcmp(type, "[shortcut]") == 0)
    {
        return SHORTCUT;
    }

    if (strcmp(type, "[scale_channels]") == 0)
    {
        return SCALE_CHANNELS;
    }

    if (strcmp(type, "[crop]") == 0)
    {
        return CROP;
    }

    if (strcmp(type, "[cost]") == 0)
    {
        return COST;
    }

    if (strcmp(type, "[detection]") == 0)
    {
        return DETECTION;
    }

    if (strcmp(type, "[region]") == 0)
    {
        return REGION;
    }

    if (strcmp(type, "[yolo]") == 0)
    {
        return YOLO;
    }

    if (strcmp(type, "[local]") == 0)
    {
        return LOCAL;
    }

    if (strcmp(type, "[conv]") == 0
        || strcmp(type, "[convolutional]") == 0)
    {
        return CONVOLUTIONAL;
    }

    if (strcmp(type, "[activation]") == 0)
    {
        return ACTIVE;
    }

    if (strcmp(type, "[net]") == 0
        || strcmp(type, "[network]") == 0)
    {
        return NETWORK;
    }

    if (strcmp(type, "[crnn]") == 0)
    {
        return CRNN;
    }

    if (strcmp(type, "[gru]") == 0)
    {
        return GRU;
    }

    if (strcmp(type, "[lstm]") == 0)
    {
        return LSTM;
    }

    if (strcmp(type, "[conv_lstm]") == 0)
    {
        return CONV_LSTM;
    }

    if (strcmp(type, "[rnn]") == 0)
    {
        return RNN;
    }

    if (strcmp(type, "[conn]") == 0
        || strcmp(type, "[connected]") == 0)
    {
        return CONNECTED;
    }

    if (strcmp(type, "[max]") == 0
        || strcmp(type, "[maxpool]") == 0)
    {
        return MAXPOOL;
    }

    if (strcmp(type, "[reorg3d]") == 0)
    {
        return REORG;
    }

    if (strcmp(type, "[reorg]") == 0)
    {
        return REORG_OLD;
    }

    if (strcmp(type, "[avg]") == 0
        || strcmp(type, "[avgpool]") == 0)
    {
        return AVGPOOL;
    }

    if (strcmp(type, "[dropout]") == 0)
    {
        return DROPOUT;
    }

    if (strcmp(type, "[lrn]") == 0
        || strcmp(type, "[normalization]") == 0)
    {
        return NORMALIZATION;
    }

    if (strcmp(type, "[batchnorm]") == 0)
    {
        return BATCHNORM;
    }

    if (strcmp(type, "[soft]") == 0
        || strcmp(type, "[softmax]") == 0)
    {
        return SOFTMAX;
    }

    if (strcmp(type, "[route]") == 0)
    {
        return ROUTE;
    }

    if (strcmp(type, "[upsample]") == 0)
    {
        return UPSAMPLE;
    }

    if (strcmp(type, "[empty]") == 0)
    {
        return EMPTY;
    }

    return BLANK;
}

void free_section(section *s)
{
    free(s->type);
    node *n = s->options->front;

    while (n)
    {
        kvp *pair = (kvp *)n->val;
        free(pair->key);
        free(pair);
        node *next = n->next;
        free(n);
        n = next;
    }

    free(s->options);
    free(s);
}

void parse_data(char *data, float *a, int n)
{
    int i;

    if (!data)
    {
        return;
    }

    char *curr = data;
    char *next = data;
    int done = 0;

    for (i = 0; i < n && !done; ++i)
    {
        while (*++next != '\0' && *next != ',');

        if (*next == '\0')
        {
            done = 1;
        }

        *next = '\0';
        sscanf(curr, "%g", &a[i]);
        curr = next + 1;
    }
}

typedef struct size_params
{
    int batch;
    int inputs;
    int h;
    int w;
    int c;
    int index;
    int time_steps;
    network net;
} size_params;

local_layer parse_local(list *options, size_params params)
{
    int n = option_find_int(options, "filters", 1);
    int size = option_find_int(options, "size", 1);
    int stride = option_find_int(options, "stride", 1);
    int pad = option_find_int(options, "pad", 0);
    char *activation_s = option_find_str(options, "activation", "logistic");
    ACTIVATION activation = get_activation(activation_s);

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before local layer must output image.");
    }

    local_layer layer = make_local_layer(batch, h, w, c, n, size, stride, pad, activation);

    return layer;
}

convolutional_layer parse_convolutional(list *options, size_params params, network net)
{
    int n = option_find_int(options, "filters", 1);
    int groups = option_find_int_quiet(options, "groups", 1);
    int size = option_find_int(options, "size", 1);
    int stride = option_find_int(options, "stride", 1);
    int dilation = option_find_int_quiet(options, "dilation", 1);

    if (size == 1)
    {
        dilation = 1;
    }

    int pad = option_find_int_quiet(options, "pad", 0);
    int padding = option_find_int_quiet(options, "padding", 0);

    if (pad)
    {
        padding = size / 2;
    }

    char *activation_s = option_find_str(options, "activation", "logistic");
    ACTIVATION activation = get_activation(activation_s);

    int share_index = option_find_int_quiet(options, "share_index", -1);
    convolutional_layer *share_layer = NULL;

    if (share_layer > -1)
    {
        share_layer = &net.layers[share_index];
    }

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before convolutional layer must output image.");
    }

    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);
    int binary = option_find_int_quiet(options, "binary", 0);
    int xnor = option_find_int_quiet(options, "xnor", 0);
    int use_bin_output = option_find_int_quiet(options, "bin_output", 0);

    convolutional_layer layer = make_convolutional_layer(batch, 1, h, w, c, n, groups, size, stride, dilation, padding, activation, batch_normalize, binary, xnor, params.net.adam, use_bin_output, params.index, share_layer);
    layer.flipped = option_find_int_quiet(options, "flipped", 0);
    layer.dot = option_find_float_quiet(options, "dot", 0);

    if (params.net.adam)
    {
        layer.B1 = params.net.B1;
        layer.B2 = params.net.B2;
        layer.eps = params.net.eps;
    }

    return layer;
}

layer parse_crnn(list *options, size_params params)
{
    int size = option_find_int_quiet(options, "size", 3);
    int stride = option_find_int_quiet(options, "stride", 1);
    int dilation = option_find_int_quiet(options, "dilation", 1);
    int pad = option_find_int_quiet(options, "pad", 0);
    int padding = option_find_int_quiet(options, "padding", 0);

    if (pad)
    {
        padding = size / 2;
    }

    int output_filters = option_find_int(options, "output", 1);
    int hidden_filters = option_find_int(options, "hidden", 1);
    int groups = option_find_int_quiet(options, "groups", 1);
    char *activation_s = option_find_str(options, "activation", "logistic");
    ACTIVATION activation = get_activation(activation_s);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);
    int xnor = option_find_int_quiet(options, "xnor", 0);

    layer l = make_crnn_layer(params.batch, params.h, params.w, params.c, hidden_filters, output_filters, groups, params.time_steps, size, stride, dilation, padding, activation, batch_normalize, xnor);

    l.shortcut = option_find_int_quiet(options, "shortcut", 0);

    return l;
}

layer parse_rnn(list *options, size_params params)
{
    int output = option_find_int(options, "output", 1);
    int hidden = option_find_int(options, "hidden", 1);
    char *activation_s = option_find_str(options, "activation", "logistic");
    ACTIVATION activation = get_activation(activation_s);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);
    int logistic = option_find_int_quiet(options, "logistic", 0);

    layer l = make_rnn_layer(params.batch, params.inputs, hidden, output, params.time_steps, activation, batch_normalize, logistic);

    l.shortcut = option_find_int_quiet(options, "shortcut", 0);

    return l;
}

layer parse_gru(list *options, size_params params)
{
    int output = option_find_int(options, "output", 1);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);

    layer l = make_gru_layer(params.batch, params.inputs, output, params.time_steps, batch_normalize);

    return l;
}

layer parse_lstm(list *options, size_params params)
{
    int output = option_find_int(options, "output", 1);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);

    layer l = make_lstm_layer(params.batch, params.inputs, output, params.time_steps, batch_normalize);

    return l;
}

layer parse_conv_lstm(list *options, size_params params)
{
    // a ConvLSTM with a larger transitional kernel should be able to capture faster motions
    int size = option_find_int_quiet(options, "size", 3);
    int stride = option_find_int_quiet(options, "stride", 1);
    int dilation = option_find_int_quiet(options, "dilation", 1);
    int pad = option_find_int_quiet(options, "pad", 0);
    int padding = option_find_int_quiet(options, "padding", 0);

    if (pad)
    {
        padding = size / 2;
    }

    int output_filters = option_find_int(options, "output", 1);
    int groups = option_find_int_quiet(options, "groups", 1);
    char *activation_s = option_find_str(options, "activation", "LINEAR");
    ACTIVATION activation = get_activation(activation_s);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);
    int xnor = option_find_int_quiet(options, "xnor", 0);
    int peephole = option_find_int_quiet(options, "peephole", 0);

    layer l = make_conv_lstm_layer(params.batch, params.h, params.w, params.c, output_filters, groups, params.time_steps, size, stride, dilation, padding, activation, batch_normalize, peephole, xnor);

    l.state_constrain = option_find_int_quiet(options, "state_constrain", params.time_steps * 32);
    l.shortcut = option_find_int_quiet(options, "shortcut", 0);

    return l;
}

connected_layer parse_connected(list *options, size_params params)
{
    int output = option_find_int(options, "output", 1);
    char *activation_s = option_find_str(options, "activation", "logistic");
    ACTIVATION activation = get_activation(activation_s);
    int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);

    connected_layer layer = make_connected_layer(params.batch, 1, params.inputs, output, activation, batch_normalize);

    return layer;
}

softmax_layer parse_softmax(list *options, size_params params)
{
    int groups = option_find_int_quiet(options, "groups", 1);
    softmax_layer layer = make_softmax_layer(params.batch, params.inputs, groups);
    layer.temperature = option_find_float_quiet(options, "temperature", 1);
    char *tree_file = option_find_str(options, "tree", 0);

    if (tree_file)
    {
        layer.softmax_tree = read_tree(tree_file);
    }

    layer.w = params.w;
    layer.h = params.h;
    layer.c = params.c;
    layer.spatial = option_find_float_quiet(options, "spatial", 0);
    layer.noloss = option_find_int_quiet(options, "noloss", 0);
    return layer;
}

int *parse_yolo_mask(char *a, int *num)
{
    int *mask = 0;

    if (a)
    {
        int len = strlen(a);
        int n = 1;
        int i;

        for (i = 0; i < len; ++i)
        {
            if (a[i] == ',')
            {
                ++n;
            }
        }

        mask = (int *)calloc(n, sizeof(int));

        for (i = 0; i < n; ++i)
        {
            int val = atoi(a);
            mask[i] = val;
            a = strchr(a, ',') + 1;
        }

        *num = n;
    }

    return mask;
}

layer parse_yolo(list *options, size_params params)
{
    int classes = option_find_int(options, "classes", 20);
    int total = option_find_int(options, "num", 1);
    int num = total;

    char *a = option_find_str(options, "mask", 0);
    int *mask = parse_yolo_mask(a, &num);
    int max_boxes = option_find_int_quiet(options, "max", 90);
    layer l = make_yolo_layer(params.batch, params.w, params.h, num, total, mask, classes, max_boxes);

    if (l.outputs != params.inputs)
    {
        printf("Error: l.outputs == params.inputs \n");
        printf("filters= in the [convolutional]-layer doesn't correspond to classes= or mask= in [yolo]-layer \n");
        exit(EXIT_FAILURE);
    }

    //assert(l.outputs == params.inputs);

    l.scale_x_y = option_find_float_quiet(options, "scale_x_y", 1);
    l.iou_normalizer = option_find_float_quiet(options, "iou_normalizer", 0.75);
    l.cls_normalizer = option_find_float_quiet(options, "cls_normalizer", 1);
    char *iou_loss = option_find_str_quiet(options, "iou_loss", "mse");   //  "iou");

    if (strcmp(iou_loss, "mse") == 0)
    {
        l.iou_loss = MSE;
    }
    else if (strcmp(iou_loss, "giou") == 0)
    {
        l.iou_loss = GIOU;
    }
    else
    {
        l.iou_loss = IOU;
    }

    fprintf(stderr, "[yolo] params: iou loss: %s, iou_norm: %2.2f, cls_norm: %2.2f, scale_x_y: %2.2f\n", (l.iou_loss == MSE ? "mse" : (l.iou_loss == GIOU ? "giou" : "iou")), l.iou_normalizer, l.cls_normalizer, l.scale_x_y);

    l.jitter = option_find_float(options, "jitter", .2);
    l.focal_loss = option_find_int_quiet(options, "focal_loss", 0);

    l.ignore_thresh = option_find_float(options, "ignore_thresh", .5);
    l.truth_thresh = option_find_float(options, "truth_thresh", 1);
    l.random = option_find_int_quiet(options, "random", 0);

    char *map_file = option_find_str(options, "map", 0);

    if (map_file)
    {
        l.map = read_map(map_file);
    }

    a = option_find_str(options, "anchors", 0);

    if (a)
    {
        int len = strlen(a);
        int n = 1;
        int i;

        for (i = 0; i < len; ++i)
        {
            if (a[i] == ',')
            {
                ++n;
            }
        }

        for (i = 0; i < n && i < total * 2; ++i)
        {
            float bias = atof(a);
            l.biases[i] = bias;
            a = strchr(a, ',') + 1;
        }
    }

    return l;
}

layer parse_region(list *options, size_params params)
{
    int coords = option_find_int(options, "coords", 4);
    int classes = option_find_int(options, "classes", 20);
    int num = option_find_int(options, "num", 1);
    int max_boxes = option_find_int_quiet(options, "max", 90);

    layer l = make_region_layer(params.batch, params.w, params.h, num, classes, coords, max_boxes);

    if (l.outputs != params.inputs)
    {
        printf("Error: l.outputs == params.inputs \n");
        printf("filters= in the [convolutional]-layer doesn't correspond to classes= or num= in [region]-layer \n");
        exit(EXIT_FAILURE);
    }

    //assert(l.outputs == params.inputs);

    l.log = option_find_int_quiet(options, "log", 0);
    l.sqrt = option_find_int_quiet(options, "sqrt", 0);

    l.softmax = option_find_int(options, "softmax", 0);
    l.focal_loss = option_find_int_quiet(options, "focal_loss", 0);
    //l.max_boxes = option_find_int_quiet(options, "max",30);
    l.jitter = option_find_float(options, "jitter", .2);
    l.rescore = option_find_int_quiet(options, "rescore", 0);

    l.thresh = option_find_float(options, "thresh", .5);
    l.classfix = option_find_int_quiet(options, "classfix", 0);
    l.absolute = option_find_int_quiet(options, "absolute", 0);
    l.random = option_find_int_quiet(options, "random", 0);

    l.coord_scale = option_find_float(options, "coord_scale", 1);
    l.object_scale = option_find_float(options, "object_scale", 1);
    l.noobject_scale = option_find_float(options, "noobject_scale", 1);
    l.mask_scale = option_find_float(options, "mask_scale", 1);
    l.class_scale = option_find_float(options, "class_scale", 1);
    l.bias_match = option_find_int_quiet(options, "bias_match", 0);

    char *tree_file = option_find_str(options, "tree", 0);

    if (tree_file)
    {
        l.softmax_tree = read_tree(tree_file);
    }

    char *map_file = option_find_str(options, "map", 0);

    if (map_file)
    {
        l.map = read_map(map_file);
    }

    char *a = option_find_str(options, "anchors", 0);

    if (a)
    {
        int len = strlen(a);
        int n = 1;
        int i;

        for (i = 0; i < len; ++i)
        {
            if (a[i] == ',')
            {
                ++n;
            }
        }

        for (i = 0; i < n && i < num * 2; ++i)
        {
            float bias = atof(a);
            l.biases[i] = bias;
            a = strchr(a, ',') + 1;
        }
    }

    return l;
}
detection_layer parse_detection(list *options, size_params params)
{
    int coords = option_find_int(options, "coords", 1);
    int classes = option_find_int(options, "classes", 1);
    int rescore = option_find_int(options, "rescore", 0);
    int num = option_find_int(options, "num", 1);
    int side = option_find_int(options, "side", 7);
    detection_layer layer = make_detection_layer(params.batch, params.inputs, num, side, classes, coords, rescore);

    layer.softmax = option_find_int(options, "softmax", 0);
    layer.sqrt = option_find_int(options, "sqrt", 0);

    layer.max_boxes = option_find_int_quiet(options, "max", 30);
    layer.coord_scale = option_find_float(options, "coord_scale", 1);
    layer.forced = option_find_int(options, "forced", 0);
    layer.object_scale = option_find_float(options, "object_scale", 1);
    layer.noobject_scale = option_find_float(options, "noobject_scale", 1);
    layer.class_scale = option_find_float(options, "class_scale", 1);
    layer.jitter = option_find_float(options, "jitter", .2);
    layer.random = option_find_int_quiet(options, "random", 0);
    layer.reorg = option_find_int_quiet(options, "reorg", 0);
    return layer;
}

cost_layer parse_cost(list *options, size_params params)
{
    char *type_s = option_find_str(options, "type", "sse");
    COST_TYPE type = get_cost_type(type_s);
    float scale = option_find_float_quiet(options, "scale", 1);
    cost_layer layer = make_cost_layer(params.batch, params.inputs, type, scale);
    layer.ratio =  option_find_float_quiet(options, "ratio", 0);
    return layer;
}

crop_layer parse_crop(list *options, size_params params)
{
    int crop_height = option_find_int(options, "crop_height", 1);
    int crop_width = option_find_int(options, "crop_width", 1);
    int flip = option_find_int(options, "flip", 0);
    float angle = option_find_float(options, "angle", 0);
    float saturation = option_find_float(options, "saturation", 1);
    float exposure = option_find_float(options, "exposure", 1);

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before crop layer must output image.");
    }

    int noadjust = option_find_int_quiet(options, "noadjust", 0);

    crop_layer l = make_crop_layer(batch, h, w, c, crop_height, crop_width, flip, angle, saturation, exposure);
    l.shift = option_find_float(options, "shift", 0);
    l.noadjust = noadjust;
    return l;
}

layer parse_reorg(list *options, size_params params)
{
    int stride = option_find_int(options, "stride", 1);
    int reverse = option_find_int_quiet(options, "reverse", 0);

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before reorg layer must output image.");
    }

    layer layer = make_reorg_layer(batch, w, h, c, stride, reverse);
    return layer;
}

layer parse_reorg_old(list *options, size_params params)
{
    printf("\n reorg_old \n");
    int stride = option_find_int(options, "stride", 1);
    int reverse = option_find_int_quiet(options, "reverse", 0);

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before reorg layer must output image.");
    }

    layer layer = make_reorg_old_layer(batch, w, h, c, stride, reverse);
    return layer;
}

maxpool_layer parse_maxpool(list *options, size_params params)
{
    int stride = option_find_int(options, "stride", 1);
    int size = option_find_int(options, "size", stride);
    int padding = option_find_int_quiet(options, "padding", size - 1);
    int maxpool_depth = option_find_int_quiet(options, "maxpool_depth", 0);
    int out_channels = option_find_int_quiet(options, "out_channels", 1);

    int batch, h, w, c;
    h = params.h;
    w = params.w;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before maxpool layer must output image.");
    }

    maxpool_layer layer = make_maxpool_layer(batch, h, w, c, size, stride, padding, maxpool_depth, out_channels);
    return layer;
}

avgpool_layer parse_avgpool(list *options, size_params params)
{
    int batch, w, h, c;
    w = params.w;
    h = params.h;
    c = params.c;
    batch = params.batch;

    if (!(h && w && c))
    {
        error("Layer before avgpool layer must output image.");
    }

    avgpool_layer layer = make_avgpool_layer(batch, w, h, c);
    return layer;
}

dropout_layer parse_dropout(list *options, size_params params)
{
    float probability = option_find_float(options, "probability", .5);
    dropout_layer layer = make_dropout_layer(params.batch, params.inputs, probability);
    layer.out_w = params.w;
    layer.out_h = params.h;
    layer.out_c = params.c;
    return layer;
}

layer parse_normalization(list *options, size_params params)
{
    float alpha = option_find_float(options, "alpha", .0001);
    float beta =  option_find_float(options, "beta" , .75);
    float kappa = option_find_float(options, "kappa", 1);
    int size = option_find_int(options, "size", 5);
    layer l = make_normalization_layer(params.batch, params.w, params.h, params.c, size, alpha, beta, kappa);
    return l;
}

layer parse_batchnorm(list *options, size_params params)
{
    layer l = make_batchnorm_layer(params.batch, params.w, params.h, params.c);
    return l;
}

layer parse_shortcut(list *options, size_params params, network net)
{
    char *l = option_find(options, "from");
    int index = atoi(l);

    if (index < 0)
    {
        index = params.index + index;
    }

    int batch = params.batch;
    layer from = net.layers[index];

    layer s = make_shortcut_layer(batch, index, params.w, params.h, params.c, from.out_w, from.out_h, from.out_c);

    char *activation_s = option_find_str(options, "activation", "linear");
    ACTIVATION activation = get_activation(activation_s);
    s.activation = activation;
    return s;
}


layer parse_scale_channels(list *options, size_params params, network net)
{
    char *l = option_find(options, "from");
    int index = atoi(l);

    if (index < 0)
    {
        index = params.index + index;
    }

    int batch = params.batch;
    layer from = net.layers[index];

    layer s = make_scale_channels_layer(batch, index, params.w, params.h, params.c, from.out_w, from.out_h, from.out_c);

    char *activation_s = option_find_str_quiet(options, "activation", "linear");
    ACTIVATION activation = get_activation(activation_s);
    s.activation = activation;
    return s;
}


layer parse_activation(list *options, size_params params)
{
    char *activation_s = option_find_str(options, "activation", "linear");
    ACTIVATION activation = get_activation(activation_s);

    layer l = make_activation_layer(params.batch, params.inputs, activation);

    l.out_h = params.h;
    l.out_w = params.w;
    l.out_c = params.c;
    l.h = params.h;
    l.w = params.w;
    l.c = params.c;

    return l;
}

layer parse_upsample(list *options, size_params params, network net)
{

    int stride = option_find_int(options, "stride", 2);
    layer l = make_upsample_layer(params.batch, params.w, params.h, params.c, stride);
    l.scale = option_find_float_quiet(options, "scale", 1);
    return l;
}

route_layer parse_route(list *options, size_params params, network net)
{
    char *l = option_find(options, "layers");
    int len = strlen(l);

    if (!l)
    {
        error("Route Layer must specify input layers");
    }

    int n = 1;
    int i;

    for (i = 0; i < len; ++i)
    {
        if (l[i] == ',')
        {
            ++n;
        }
    }

    int *layers = (int *)calloc(n, sizeof(int));
    int *sizes = (int *)calloc(n, sizeof(int));

    for (i = 0; i < n; ++i)
    {
        int index = atoi(l);
        l = strchr(l, ',') + 1;

        if (index < 0)
        {
            index = params.index + index;
        }

        layers[i] = index;
        sizes[i] = net.layers[index].outputs;
    }

    int batch = params.batch;

    route_layer layer = make_route_layer(batch, n, layers, sizes);

    convolutional_layer first = net.layers[layers[0]];
    layer.out_w = first.out_w;
    layer.out_h = first.out_h;
    layer.out_c = first.out_c;

    for (i = 1; i < n; ++i)
    {
        int index = layers[i];
        convolutional_layer next = net.layers[index];

        if (next.out_w == first.out_w && next.out_h == first.out_h)
        {
            layer.out_c += next.out_c;
        }
        else
        {
            layer.out_h = layer.out_w = layer.out_c = 0;
        }
    }

    return layer;
}

learning_rate_policy get_policy(char *s)
{
    if (strcmp(s, "random") == 0)
    {
        return RANDOM;
    }

    if (strcmp(s, "poly") == 0)
    {
        return POLY;
    }

    if (strcmp(s, "constant") == 0)
    {
        return CONSTANT;
    }

    if (strcmp(s, "step") == 0)
    {
        return STEP;
    }

    if (strcmp(s, "exp") == 0)
    {
        return EXP;
    }

    if (strcmp(s, "sigmoid") == 0)
    {
        return SIG;
    }

    if (strcmp(s, "steps") == 0)
    {
        return STEPS;
    }

    if (strcmp(s, "sgdr") == 0)
    {
        return SGDR;
    }

    fprintf(stderr, "Couldn't find policy %s, going with constant\n", s);
    return CONSTANT;
}

void parse_net_options(list *options, network *net)
{
    net->batch = option_find_int(options, "batch", 1);
    net->learning_rate = option_find_float(options, "learning_rate", .001);
    net->learning_rate_min = option_find_float_quiet(options, "learning_rate_min", .00001);
    net->batches_per_cycle = option_find_int_quiet(options, "sgdr_cycle", 1000);
    net->batches_cycle_mult = option_find_int_quiet(options, "sgdr_mult", 2);
    net->momentum = option_find_float(options, "momentum", .9);
    net->decay = option_find_float(options, "decay", .0001);
    int subdivs = option_find_int(options, "subdivisions", 1);
    net->time_steps = option_find_int_quiet(options, "time_steps", 1);
    net->track = option_find_int_quiet(options, "track", 0);
    net->augment_speed = option_find_int_quiet(options, "augment_speed", 2);
    net->init_sequential_subdivisions = net->sequential_subdivisions = option_find_int_quiet(options, "sequential_subdivisions", subdivs);

    if (net->sequential_subdivisions > subdivs)
    {
        net->init_sequential_subdivisions = net->sequential_subdivisions = subdivs;
    }

    net->try_fix_nan = option_find_int_quiet(options, "try_fix_nan", 0);
    net->batch /= subdivs;
    net->batch *= net->time_steps;
    net->subdivisions = subdivs;

    net->adam = option_find_int_quiet(options, "adam", 0);

    if (net->adam)
    {
        net->B1 = option_find_float(options, "B1", .9);
        net->B2 = option_find_float(options, "B2", .999);
        net->eps = option_find_float(options, "eps", .000001);
    }

    net->h = option_find_int_quiet(options, "height", 0);
    net->w = option_find_int_quiet(options, "width", 0);
    net->c = option_find_int_quiet(options, "channels", 0);
    net->inputs = option_find_int_quiet(options, "inputs", net->h * net->w * net->c);
    net->max_crop = option_find_int_quiet(options, "max_crop", net->w * 2);
    net->min_crop = option_find_int_quiet(options, "min_crop", net->w);
    net->flip = option_find_int_quiet(options, "flip", 1);
    net->blur = option_find_int_quiet(options, "blur", 0);
    net->mixup = option_find_int_quiet(options, "mixup", 0);
    net->letter_box = option_find_int_quiet(options, "letter_box", 0);

    net->angle = option_find_float_quiet(options, "angle", 0);
    net->aspect = option_find_float_quiet(options, "aspect", 1);
    net->saturation = option_find_float_quiet(options, "saturation", 1);
    net->exposure = option_find_float_quiet(options, "exposure", 1);
    net->hue = option_find_float_quiet(options, "hue", 0);
    net->power = option_find_float_quiet(options, "power", 4);

    if (!net->inputs && !(net->h && net->w && net->c))
    {
        error("No input parameters supplied");
    }

    char *policy_s = option_find_str(options, "policy", "constant");
    net->policy = get_policy(policy_s);
    net->burn_in = option_find_int_quiet(options, "burn_in", 0);
#ifdef CUDNN_HALF

    if (net->gpu_index >= 0)
    {
        int compute_capability = get_gpu_compute_capability(net->gpu_index);

        if (get_gpu_compute_capability(net->gpu_index) >= 700)
        {
            net->cudnn_half = 1;
        }
        else
        {
            net->cudnn_half = 0;
        }

        fprintf(stderr, " compute_capability = %d, cudnn_half = %d \n", compute_capability, net->cudnn_half);
    }
    else
    {
        fprintf(stderr, " GPU isn't used \n");
    }

#endif

    if (net->policy == STEP)
    {
        net->step = option_find_int(options, "step", 1);
        net->scale = option_find_float(options, "scale", 1);
    }
    else if (net->policy == STEPS || net->policy == SGDR)
    {
        char *l = option_find(options, "steps");
        char *p = option_find(options, "scales");
        char *s = option_find(options, "seq_scales");

        if (net->policy == STEPS && (!l || !p))
        {
            error("STEPS policy must have steps and scales in cfg file");
        }

        if (l)
        {
            int len = strlen(l);
            int n = 1;
            int i;

            for (i = 0; i < len; ++i)
            {
                if (l[i] == ',')
                {
                    ++n;
                }
            }

            int *steps = (int *)calloc(n, sizeof(int));
            float *scales = (float *)calloc(n, sizeof(float));
            float *seq_scales = (float *)calloc(n, sizeof(float));

            for (i = 0; i < n; ++i)
            {
                float scale = 1.0;

                if (p)
                {
                    scale = atof(p);
                    p = strchr(p, ',') + 1;
                }

                float sequence_scale = 1.0;

                if (s)
                {
                    sequence_scale = atof(s);
                    s = strchr(s, ',') + 1;
                }

                int step = atoi(l);
                l = strchr(l, ',') + 1;
                steps[i] = step;
                scales[i] = scale;
                seq_scales[i] = sequence_scale;
            }

            net->scales = scales;
            net->steps = steps;
            net->seq_scales = seq_scales;
            net->num_steps = n;
        }
    }
    else if (net->policy == EXP)
    {
        net->gamma = option_find_float(options, "gamma", 1);
    }
    else if (net->policy == SIG)
    {
        net->gamma = option_find_float(options, "gamma", 1);
        net->step = option_find_int(options, "step", 1);
    }
    else if (net->policy == POLY || net->policy == RANDOM)
    {
        //net->power = option_find_float(options, "power", 1);
    }

    net->max_batches = option_find_int(options, "max_batches", 0);
}

int is_network(section *s)
{
    return (strcmp(s->type, "[net]") == 0
            || strcmp(s->type, "[network]") == 0);
}

network parse_network_cfg(char *filename)
{
    return parse_network_cfg_custom(filename, 0, 0);
}

network parse_network_cfg_custom(char *filename, int batch, int time_steps)
{
    list *sections = read_cfg(filename);
    node *n = sections->front;

    if (!n)
    {
        error("Config file has no sections");
    }

    network net = make_network(sections->size - 1);
    net.gpu_index = gpu_index;
    size_params params;

    section *s = (section *)n->val;
    list *options = s->options;

    if (!is_network(s))
    {
        error("First section must be [net] or [network]");
    }

    parse_net_options(options, &net);

    params.h = net.h;
    params.w = net.w;
    params.c = net.c;
    params.inputs = net.inputs;

    if (batch > 0)
    {
        net.batch = batch;
    }

    if (time_steps > 0)
    {
        net.time_steps = time_steps;
    }

    if (net.batch < net.time_steps)
    {
        net.batch = net.time_steps;
    }

    params.batch = net.batch;
    params.time_steps = net.time_steps;
    params.net = net;

    float bflops = 0;
    size_t workspace_size = 0;
    size_t max_inputs = 0;
    size_t max_outputs = 0;
    n = n->next;
    int count = 0;
    free_section(s);
    fprintf(stderr, "   layer   filters  size/strd(dil)      input                output\n");

    while (n)
    {
        params.index = count;
        fprintf(stderr, "%4d ", count);
        s = (section *)n->val;
        options = s->options;
        layer l = { (LAYER_TYPE)0 };
        LAYER_TYPE lt = string_to_layer_type(s->type);

        if (lt == CONVOLUTIONAL)
        {
            l = parse_convolutional(options, params, net);
        }
        else if (lt == LOCAL)
        {
            l = parse_local(options, params);
        }
        else if (lt == ACTIVE)
        {
            l = parse_activation(options, params);
        }
        else if (lt == RNN)
        {
            l = parse_rnn(options, params);
        }
        else if (lt == GRU)
        {
            l = parse_gru(options, params);
        }
        else if (lt == LSTM)
        {
            l = parse_lstm(options, params);
        }
        else if (lt == CONV_LSTM)
        {
            l = parse_conv_lstm(options, params);
        }
        else if (lt == CRNN)
        {
            l = parse_crnn(options, params);
        }
        else if (lt == CONNECTED)
        {
            l = parse_connected(options, params);
        }
        else if (lt == CROP)
        {
            l = parse_crop(options, params);
        }
        else if (lt == COST)
        {
            l = parse_cost(options, params);
        }
        else if (lt == REGION)
        {
            l = parse_region(options, params);
        }
        else if (lt == YOLO)
        {
            l = parse_yolo(options, params);
        }
        else if (lt == DETECTION)
        {
            l = parse_detection(options, params);
        }
        else if (lt == SOFTMAX)
        {
            l = parse_softmax(options, params);
            net.hierarchy = l.softmax_tree;
        }
        else if (lt == NORMALIZATION)
        {
            l = parse_normalization(options, params);
        }
        else if (lt == BATCHNORM)
        {
            l = parse_batchnorm(options, params);
        }
        else if (lt == MAXPOOL)
        {
            l = parse_maxpool(options, params);
        }
        else if (lt == REORG)
        {
            l = parse_reorg(options, params);
        }
        else if (lt == REORG_OLD)
        {
            l = parse_reorg_old(options, params);
        }
        else if (lt == AVGPOOL)
        {
            l = parse_avgpool(options, params);
        }
        else if (lt == ROUTE)
        {
            l = parse_route(options, params, net);
            int k;

            for (k = 0; k < l.n; ++k)
            {
                net.layers[l.input_layers[k]].use_bin_output = 0;
            }
        }
        else if (lt == UPSAMPLE)
        {
            l = parse_upsample(options, params, net);
        }
        else if (lt == SHORTCUT)
        {
            l = parse_shortcut(options, params, net);
            net.layers[count - 1].use_bin_output = 0;
            net.layers[l.index].use_bin_output = 0;
        }
        else if (lt == SCALE_CHANNELS)
        {
            l = parse_scale_channels(options, params, net);
            net.layers[count - 1].use_bin_output = 0;
            net.layers[l.index].use_bin_output = 0;
        }
        else if (lt == DROPOUT)
        {
            l = parse_dropout(options, params);
            l.output = net.layers[count - 1].output;
            l.delta = net.layers[count - 1].delta;
#ifdef GPU
            l.output_gpu = net.layers[count - 1].output_gpu;
            l.delta_gpu = net.layers[count - 1].delta_gpu;
#endif
        }
        else if (lt == EMPTY)
        {
            layer empty_layer;
            empty_layer.out_w = params.w;
            empty_layer.out_h = params.h;
            empty_layer.out_c = params.c;
            l = empty_layer;
            l.output = net.layers[count - 1].output;
            l.delta = net.layers[count - 1].delta;
#ifdef GPU
            l.output_gpu = net.layers[count - 1].output_gpu;
            l.delta_gpu = net.layers[count - 1].delta_gpu;
#endif
        }
        else
        {
            fprintf(stderr, "Type not recognized: %s\n", s->type);
        }

        l.onlyforward = option_find_int_quiet(options, "onlyforward", 0);
        l.stopbackward = option_find_int_quiet(options, "stopbackward", 0);
        l.dontload = option_find_int_quiet(options, "dontload", 0);
        l.dontloadscales = option_find_int_quiet(options, "dontloadscales", 0);
        l.learning_rate_scale = option_find_float_quiet(options, "learning_rate", 1);
        option_unused(options);
        net.layers[count] = l;

        if (l.workspace_size > workspace_size)
        {
            workspace_size = l.workspace_size;
        }

        if (l.inputs > max_inputs)
        {
            max_inputs = l.inputs;
        }

        if (l.outputs > max_outputs)
        {
            max_outputs = l.outputs;
        }

        free_section(s);
        n = n->next;
        ++count;

        if (n)
        {
            params.h = l.out_h;
            params.w = l.out_w;
            params.c = l.out_c;
            params.inputs = l.outputs;
        }

        if (l.bflops > 0)
        {
            bflops += l.bflops;
        }
    }

    free_list(sections);
    net.outputs = get_network_output_size(net);
    net.output = get_network_output(net);
    fprintf(stderr, "Total BFLOPS %5.3f \n", bflops);
#ifdef GPU
    get_cuda_stream();
    get_cuda_memcpy_stream();

    if (gpu_index >= 0)
    {
        int size = get_network_input_size(net) * net.batch;
        net.input_state_gpu = cuda_make_array(0, size);

        if (cudaSuccess == cudaHostAlloc(&net.input_pinned_cpu, size * sizeof(float), cudaHostRegisterMapped))
        {
            net.input_pinned_cpu_flag = 1;
        }
        else
        {
            cudaGetLastError(); // reset CUDA-error
            net.input_pinned_cpu = (float *)calloc(size, sizeof(float));
        }

        // pre-allocate memory for inference on Tensor Cores (fp16)
        if (net.cudnn_half)
        {
            *net.max_input16_size = max_inputs;
            CHECK_CUDA(cudaMalloc((void **)net.input16_gpu, *net.max_input16_size * sizeof(short))); //sizeof(half)
            *net.max_output16_size = max_outputs;
            CHECK_CUDA(cudaMalloc((void **)net.output16_gpu, *net.max_output16_size * sizeof(short))); //sizeof(half)
        }

        if (workspace_size)
        {
            fprintf(stderr, " Allocate additional workspace_size = %1.2f MB \n", (float)workspace_size / 1000000);
            net.workspace = cuda_make_array(0, workspace_size / sizeof(float) + 1);
        }
        else
        {
            net.workspace = (float *)calloc(1, workspace_size);
        }
    }

#else

    if (workspace_size)
    {
        net.workspace = (float *)calloc(1, workspace_size);
    }

#endif

    LAYER_TYPE lt = net.layers[net.n - 1].type;

    if ((net.w % 32 != 0 || net.h % 32 != 0) && (lt == YOLO || lt == REGION || lt == DETECTION))
    {
        printf("\n Warning: width=%d and height=%d in cfg-file must be divisible by 32 for default networks Yolo v1/v2/v3!!! \n\n",
               net.w, net.h);
    }

    return net;
}



list *read_cfg(char *filename)
{
    FILE *file = fopen(filename, "r");

    if (file == 0)
    {
        file_error(filename);
    }

    char *line;
    int nu = 0;
    list *sections = make_list();
    section *current = 0;

    while ((line = fgetl(file)) != 0)
    {
        ++ nu;
        strip(line);

        switch (line[0])
        {
            case '[':
                current = (section *)malloc(sizeof(section));
                list_insert(sections, current);
                current->options = make_list();
                current->type = line;
                break;

            case '\0':
            case '#':
            case ';':
                free(line);
                break;

            default:
                if (!read_option(line, current->options))
                {
                    fprintf(stderr, "Config file error line %d, could parse: %s\n", nu, line);
                    free(line);
                }

                break;
        }
    }

    fclose(file);
    return sections;
}

void save_convolutional_weights_binary(layer l, FILE *fp)
{
#ifdef GPU

    if (gpu_index >= 0)
    {
        pull_convolutional_layer(l);
    }

#endif
    int size = (l.c / l.groups) * l.size * l.size;
    binarize_weights(l.weights, l.n, size, l.binary_weights);
    int i, j, k;
    fwrite(l.biases, sizeof(float), l.n, fp);

    if (l.batch_normalize)
    {
        fwrite(l.scales, sizeof(float), l.n, fp);
        fwrite(l.rolling_mean, sizeof(float), l.n, fp);
        fwrite(l.rolling_variance, sizeof(float), l.n, fp);
    }

    for (i = 0; i < l.n; ++i)
    {
        float mean = l.binary_weights[i * size];

        if (mean < 0)
        {
            mean = -mean;
        }

        fwrite(&mean, sizeof(float), 1, fp);

        for (j = 0; j < size / 8; ++j)
        {
            int index = i * size + j * 8;
            unsigned char c = 0;

            for (k = 0; k < 8; ++k)
            {
                if (j * 8 + k >= size)
                {
                    break;
                }

                if (l.binary_weights[index + k] > 0)
                {
                    c = (c | 1 << k);
                }
            }

            fwrite(&c, sizeof(char), 1, fp);
        }
    }
}

void save_convolutional_weights(layer l, FILE *fp)
{
    if (l.binary)
    {
        //save_convolutional_weights_binary(l, fp);
        //return;
    }

#ifdef GPU

    if (gpu_index >= 0)
    {
        pull_convolutional_layer(l);
    }

#endif
    int num = l.nweights;
    fwrite(l.biases, sizeof(float), l.n, fp);

    if (l.batch_normalize)
    {
        fwrite(l.scales, sizeof(float), l.n, fp);
        fwrite(l.rolling_mean, sizeof(float), l.n, fp);
        fwrite(l.rolling_variance, sizeof(float), l.n, fp);
    }

    fwrite(l.weights, sizeof(float), num, fp);
    //if(l.adam){
    //    fwrite(l.m, sizeof(float), num, fp);
    //    fwrite(l.v, sizeof(float), num, fp);
    //}
}

void save_batchnorm_weights(layer l, FILE *fp)
{
#ifdef GPU

    if (gpu_index >= 0)
    {
        pull_batchnorm_layer(l);
    }

#endif
    fwrite(l.scales, sizeof(float), l.c, fp);
    fwrite(l.rolling_mean, sizeof(float), l.c, fp);
    fwrite(l.rolling_variance, sizeof(float), l.c, fp);
}

void save_connected_weights(layer l, FILE *fp)
{
#ifdef GPU

    if (gpu_index >= 0)
    {
        pull_connected_layer(l);
    }

#endif
    fwrite(l.biases, sizeof(float), l.outputs, fp);
    fwrite(l.weights, sizeof(float), l.outputs * l.inputs, fp);

    if (l.batch_normalize)
    {
        fwrite(l.scales, sizeof(float), l.outputs, fp);
        fwrite(l.rolling_mean, sizeof(float), l.outputs, fp);
        fwrite(l.rolling_variance, sizeof(float), l.outputs, fp);
    }
}

void save_weights_upto(network net, char *filename, int cutoff)
{
#ifdef GPU

    if (net.gpu_index >= 0)
    {
        cuda_set_device(net.gpu_index);
    }

#endif
    fprintf(stderr, "Saving weights to %s\n", filename);
    FILE *fp = fopen(filename, "wb");

    if (!fp)
    {
        file_error(filename);
    }

    int major = MAJOR_VERSION;
    int minor = MINOR_VERSION;
    int revision = PATCH_VERSION;
    fwrite(&major, sizeof(int), 1, fp);
    fwrite(&minor, sizeof(int), 1, fp);
    fwrite(&revision, sizeof(int), 1, fp);
    fwrite(net.seen, sizeof(uint64_t), 1, fp);

    int i;

    for (i = 0; i < net.n && i < cutoff; ++i)
    {
        layer l = net.layers[i];

        if (l.type == CONVOLUTIONAL && l.share_layer == NULL)
        {
            save_convolutional_weights(l, fp);
        }

        if (l.type == CONNECTED)
        {
            save_connected_weights(l, fp);
        }

        if (l.type == BATCHNORM)
        {
            save_batchnorm_weights(l, fp);
        }

        if (l.type == RNN)
        {
            save_connected_weights(*(l.input_layer), fp);
            save_connected_weights(*(l.self_layer), fp);
            save_connected_weights(*(l.output_layer), fp);
        }

        if (l.type == GRU)
        {
            save_connected_weights(*(l.input_z_layer), fp);
            save_connected_weights(*(l.input_r_layer), fp);
            save_connected_weights(*(l.input_h_layer), fp);
            save_connected_weights(*(l.state_z_layer), fp);
            save_connected_weights(*(l.state_r_layer), fp);
            save_connected_weights(*(l.state_h_layer), fp);
        }

        if (l.type == LSTM)
        {
            save_connected_weights(*(l.wf), fp);
            save_connected_weights(*(l.wi), fp);
            save_connected_weights(*(l.wg), fp);
            save_connected_weights(*(l.wo), fp);
            save_connected_weights(*(l.uf), fp);
            save_connected_weights(*(l.ui), fp);
            save_connected_weights(*(l.ug), fp);
            save_connected_weights(*(l.uo), fp);
        }

        if (l.type == CONV_LSTM)
        {
            if (l.peephole)
            {
                save_convolutional_weights(*(l.vf), fp);
                save_convolutional_weights(*(l.vi), fp);
                save_convolutional_weights(*(l.vo), fp);
            }

            save_convolutional_weights(*(l.wf), fp);
            save_convolutional_weights(*(l.wi), fp);
            save_convolutional_weights(*(l.wg), fp);
            save_convolutional_weights(*(l.wo), fp);
            save_convolutional_weights(*(l.uf), fp);
            save_convolutional_weights(*(l.ui), fp);
            save_convolutional_weights(*(l.ug), fp);
            save_convolutional_weights(*(l.uo), fp);
        }

        if (l.type == CRNN)
        {
            save_convolutional_weights(*(l.input_layer), fp);
            save_convolutional_weights(*(l.self_layer), fp);
            save_convolutional_weights(*(l.output_layer), fp);
        }

        if (l.type == LOCAL)
        {
#ifdef GPU

            if (gpu_index >= 0)
            {
                pull_local_layer(l);
            }

#endif
            int locations = l.out_w * l.out_h;
            int size = l.size * l.size * l.c * l.n * locations;
            fwrite(l.biases, sizeof(float), l.outputs, fp);
            fwrite(l.weights, sizeof(float), size, fp);
        }
    }

    fclose(fp);
}
void save_weights(network net, char *filename)
{
    save_weights_upto(net, filename, net.n);
}

void transpose_matrix(float *a, int rows, int cols)
{
    float *transpose = (float *)calloc(rows * cols, sizeof(float));
    int x, y;

    for (x = 0; x < rows; ++x)
    {
        for (y = 0; y < cols; ++y)
        {
            transpose[y * rows + x] = a[x * cols + y];
        }
    }

    memcpy(a, transpose, rows * cols * sizeof(float));
    free(transpose);
}

void load_connected_weights(layer l, FILE *fp, int transpose)
{
    fread(l.biases, sizeof(float), l.outputs, fp);
    fread(l.weights, sizeof(float), l.outputs * l.inputs, fp);

    if (transpose)
    {
        transpose_matrix(l.weights, l.inputs, l.outputs);
    }

    //printf("Biases: %f mean %f variance\n", mean_array(l.biases, l.outputs), variance_array(l.biases, l.outputs));
    //printf("Weights: %f mean %f variance\n", mean_array(l.weights, l.outputs*l.inputs), variance_array(l.weights, l.outputs*l.inputs));
    if (l.batch_normalize && (!l.dontloadscales))
    {
        fread(l.scales, sizeof(float), l.outputs, fp);
        fread(l.rolling_mean, sizeof(float), l.outputs, fp);
        fread(l.rolling_variance, sizeof(float), l.outputs, fp);
        //printf("Scales: %f mean %f variance\n", mean_array(l.scales, l.outputs), variance_array(l.scales, l.outputs));
        //printf("rolling_mean: %f mean %f variance\n", mean_array(l.rolling_mean, l.outputs), variance_array(l.rolling_mean, l.outputs));
        //printf("rolling_variance: %f mean %f variance\n", mean_array(l.rolling_variance, l.outputs), variance_array(l.rolling_variance, l.outputs));
    }

#ifdef GPU

    if (gpu_index >= 0)
    {
        push_connected_layer(l);
    }

#endif
}

void load_batchnorm_weights(layer l, FILE *fp)
{
    fread(l.scales, sizeof(float), l.c, fp);
    fread(l.rolling_mean, sizeof(float), l.c, fp);
    fread(l.rolling_variance, sizeof(float), l.c, fp);
#ifdef GPU

    if (gpu_index >= 0)
    {
        push_batchnorm_layer(l);
    }

#endif
}

void load_convolutional_weights_binary(layer l, FILE *fp)
{
    fread(l.biases, sizeof(float), l.n, fp);

    if (l.batch_normalize && (!l.dontloadscales))
    {
        fread(l.scales, sizeof(float), l.n, fp);
        fread(l.rolling_mean, sizeof(float), l.n, fp);
        fread(l.rolling_variance, sizeof(float), l.n, fp);
    }

    int size = (l.c / l.groups) * l.size * l.size;
    int i, j, k;

    for (i = 0; i < l.n; ++i)
    {
        float mean = 0;
        fread(&mean, sizeof(float), 1, fp);

        for (j = 0; j < size / 8; ++j)
        {
            int index = i * size + j * 8;
            unsigned char c = 0;
            fread(&c, sizeof(char), 1, fp);

            for (k = 0; k < 8; ++k)
            {
                if (j * 8 + k >= size)
                {
                    break;
                }

                l.weights[index + k] = (c & 1 << k) ? mean : -mean;
            }
        }
    }

#ifdef GPU

    if (gpu_index >= 0)
    {
        push_convolutional_layer(l);
    }

#endif
}

void load_convolutional_weights(layer l, FILE *fp)
{
    if (l.binary)
    {
        //load_convolutional_weights_binary(l, fp);
        //return;
    }

    int num = l.nweights;
    fread(l.biases, sizeof(float), l.n, fp);

    //fread(l.weights, sizeof(float), num, fp); // as in connected layer
    if (l.batch_normalize && (!l.dontloadscales))
    {
        fread(l.scales, sizeof(float), l.n, fp);
        fread(l.rolling_mean, sizeof(float), l.n, fp);
        fread(l.rolling_variance, sizeof(float), l.n, fp);

        if (0)
        {
            int i;

            for (i = 0; i < l.n; ++i)
            {
                printf("%g, ", l.rolling_mean[i]);
            }

            printf("\n");

            for (i = 0; i < l.n; ++i)
            {
                printf("%g, ", l.rolling_variance[i]);
            }

            printf("\n");
        }

        if (0)
        {
            fill_cpu(l.n, 0, l.rolling_mean, 1);
            fill_cpu(l.n, 0, l.rolling_variance, 1);
        }
    }

    fread(l.weights, sizeof(float), num, fp);

    //if(l.adam){
    //    fread(l.m, sizeof(float), num, fp);
    //    fread(l.v, sizeof(float), num, fp);
    //}
    //if(l.c == 3) scal_cpu(num, 1./256, l.weights, 1);
    if (l.flipped)
    {
        transpose_matrix(l.weights, (l.c / l.groups)*l.size * l.size, l.n);
    }

    //if (l.binary) binarize_weights(l.weights, l.n, (l.c/l.groups)*l.size*l.size, l.weights);
#ifdef GPU

    if (gpu_index >= 0)
    {
        push_convolutional_layer(l);
    }

#endif
}


void load_weights_upto(network *net, char *filename, int cutoff)
{
#ifdef GPU

    if (net->gpu_index >= 0)
    {
        cuda_set_device(net->gpu_index);
    }

#endif
    fprintf(stderr, "Loading weights from %s...", filename);
    fflush(stdout);
    FILE *fp = fopen(filename, "rb");

    if (!fp)
    {
        file_error(filename);
    }

    int major;
    int minor;
    int revision;
    fread(&major, sizeof(int), 1, fp);
    fread(&minor, sizeof(int), 1, fp);
    fread(&revision, sizeof(int), 1, fp);

    if ((major * 10 + minor) >= 2)
    {
        printf("\n seen 64 \n");
        uint64_t iseen = 0;
        fread(&iseen, sizeof(uint64_t), 1, fp);
        *net->seen = iseen;
    }
    else
    {
        printf("\n seen 32 \n");
        uint32_t iseen = 0;
        fread(&iseen, sizeof(uint32_t), 1, fp);
        *net->seen = iseen;
    }

    int transpose = (major > 1000) || (minor > 1000);

    int i;

    for (i = 0; i < net->n && i < cutoff; ++i)
    {
        layer l = net->layers[i];

        if (l.dontload)
        {
            continue;
        }

        if (l.type == CONVOLUTIONAL && l.share_layer == NULL)
        {
            load_convolutional_weights(l, fp);
        }

        if (l.type == CONNECTED)
        {
            load_connected_weights(l, fp, transpose);
        }

        if (l.type == BATCHNORM)
        {
            load_batchnorm_weights(l, fp);
        }

        if (l.type == CRNN)
        {
            load_convolutional_weights(*(l.input_layer), fp);
            load_convolutional_weights(*(l.self_layer), fp);
            load_convolutional_weights(*(l.output_layer), fp);
        }

        if (l.type == RNN)
        {
            load_connected_weights(*(l.input_layer), fp, transpose);
            load_connected_weights(*(l.self_layer), fp, transpose);
            load_connected_weights(*(l.output_layer), fp, transpose);
        }

        if (l.type == GRU)
        {
            load_connected_weights(*(l.input_z_layer), fp, transpose);
            load_connected_weights(*(l.input_r_layer), fp, transpose);
            load_connected_weights(*(l.input_h_layer), fp, transpose);
            load_connected_weights(*(l.state_z_layer), fp, transpose);
            load_connected_weights(*(l.state_r_layer), fp, transpose);
            load_connected_weights(*(l.state_h_layer), fp, transpose);
        }

        if (l.type == LSTM)
        {
            load_connected_weights(*(l.wf), fp, transpose);
            load_connected_weights(*(l.wi), fp, transpose);
            load_connected_weights(*(l.wg), fp, transpose);
            load_connected_weights(*(l.wo), fp, transpose);
            load_connected_weights(*(l.uf), fp, transpose);
            load_connected_weights(*(l.ui), fp, transpose);
            load_connected_weights(*(l.ug), fp, transpose);
            load_connected_weights(*(l.uo), fp, transpose);
        }

        if (l.type == CONV_LSTM)
        {
            if (l.peephole)
            {
                load_convolutional_weights(*(l.vf), fp);
                load_convolutional_weights(*(l.vi), fp);
                load_convolutional_weights(*(l.vo), fp);
            }

            load_convolutional_weights(*(l.wf), fp);
            load_convolutional_weights(*(l.wi), fp);
            load_convolutional_weights(*(l.wg), fp);
            load_convolutional_weights(*(l.wo), fp);
            load_convolutional_weights(*(l.uf), fp);
            load_convolutional_weights(*(l.ui), fp);
            load_convolutional_weights(*(l.ug), fp);
            load_convolutional_weights(*(l.uo), fp);
        }

        if (l.type == LOCAL)
        {
            int locations = l.out_w * l.out_h;
            int size = l.size * l.size * l.c * l.n * locations;
            fread(l.biases, sizeof(float), l.outputs, fp);
            fread(l.weights, sizeof(float), size, fp);
#ifdef GPU

            if (gpu_index >= 0)
            {
                push_local_layer(l);
            }

#endif
        }
    }

    fprintf(stderr, "Done!\n");
    fclose(fp);
}

void load_weights(network *net, char *filename)
{
    load_weights_upto(net, filename, net->n);
}

// load network & force - set batch size
network *load_network_custom(char *cfg, char *weights, int clear, int batch)
{
    printf(" Try to load cfg: %s, weights: %s, clear = %d \n", cfg, weights, clear);
    network *net = (network *)calloc(1, sizeof(network));
    *net = parse_network_cfg_custom(cfg, batch, 0);

    if (weights && weights[0] != 0)
    {
        load_weights(net, weights);
    }

    if (clear)
    {
        (*net->seen) = 0;
    }

    return net;
}

// load network & get batch size from cfg-file
network *load_network(char *cfg, char *weights, int clear)
{
    printf(" Try to load cfg: %s, weights: %s, clear = %d \n", cfg, weights, clear);
    network *net = (network *)calloc(1, sizeof(network));
    *net = parse_network_cfg(cfg);

    if (weights && weights[0] != 0)
    {
        load_weights(net, weights);
    }

    if (clear)
    {
        (*net->seen) = 0;
    }

    return net;
}
